package com.coderfans.webmaic;

import com.coderfans.logwrapper.LogUtils;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by ${fanchunshuai} on 2017-6-27.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2017 58到家-版权所有
 */
public class SpiderCsdnLink  implements PageProcessor {

    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    Logger logger = LogUtils.getLogger(SpiderCsdnLink.class);
    private File file = new File("E:\\spiderFile\\csdnSpiderLog6.log");

    //定制爬虫逻辑的核心接口，这里编写抽取逻辑
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(http://my.csdn.net/\\w+/\\w+)").all());
        page.addTargetRequests(page.getHtml().links().regex("(http://my.csdn.net/\\w+)").all());

        if(StringUtils.isNotBlank(page.getHtml().get())){
            List<String> hrefList = page.getHtml().xpath("//a/@href").all();
            System.out.println(hrefList);
            List<String> newHrefList = new ArrayList<>();

            for (String str: hrefList){
                if(StringUtils.countMatches(str,"/")==3 && str.startsWith("http://my.csdn.net/")){
                    newHrefList.add(str);
                }
            }
            System.out.println("---------------------------------");
            //String filename = page.getRequest().getUrl().replace(":","").replace("/","-");
            //System.out.println("url = " + page.getRequest().getUrl());
            try {
                Files.append(newHrefList.toString()+"\n",file, Charsets.UTF_8); //追加文件
            } catch (IOException e) {
                e.printStackTrace();
            }
           /* String filepath = "E:/githubfilelist/"+String.format("yyyy-MM-dd",new Date())+"/"+filename+".html";
            File file = new File(filepath);
            if (!file.getParentFile().exists()) {
                file.getParentFile().mkdirs();
            }
            try {
                if(!file.exists()){
                    file.createNewFile();
                    FileWriter fw = new FileWriter(file, true);
                    BufferedWriter bw = new BufferedWriter(fw);
                    bw.write(page.getHtml().get());
                    bw.flush();
                    bw.close();
                    fw.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }*/
        }
        /*if (page.getResultItems().get("name")==null){
            //设置skip之后，这个页面的结果不会被Pipeline处理
            page.setSkip(true);
        }*/
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {

        Spider.create(new SpiderCsdnLink()).addPipeline(new com.coderfans.datagithub.ConsolePipeline())
                .addUrl("http://my.csdn.net/siimon")
                .thread(8).run();


    }
}
